Testing
Necessary libraries
library(MASS)
library(ISLR)
library(ElemStatLearn)
library(agtboost)
library(ggplot2)Load functions
setwd("/Users/eirik/OneDrive - University of Bergen")
source("functions.r")Testing
Default function
Testing default model with sample size = 1 to see if it produces similar results as standard agtb
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = 1, "nrounds"=1000)
agt_models = list() #List of all agt-models, agt_models[[i]][[b]] gives dataset i, iteration b
sam_models = list() #List of all sampling models
sam_test_loss = list() #List of all test loss for sampling models
agt_test_loss = list() #List of all test loss for agt models
agt_train_loss = list() #List of all train loss for agt models
agt_estgen_loss = list() #List of all estimated generalization loss for agt models
res <- list() #List of all mse of all models for all iterations on all datasets
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7, style = 3)
sub_models = list()
sub_agt_models = list()
sub_test_loss = list()
sub_agt_test_loss = list()
sub_agt_train_loss = list()
sub_agt_estgen_loss = list()
res_mat <- matrix(nrow=B, ncol=2)
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
set.seed(seeds[b])
mod = sampling.agt.train(x.train, y.train, samSize = param$samSize, learnRate = param$learning_rate, Nrounds = param$nrounds,
verbose = 0, algorithm = "global_subset", type = "reg", force_continued_learning = F)
sub_models = c(sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_test_loss = c(sub_test_loss, list(test_mse))
set.seed(seeds[b])
agt.mod = gbt.train(y.train, x.train, param$learning_rate, nrounds = param$nrounds,
verbose = 0, algorithm = "global_subset")
sub_agt_models = c(sub_agt_models, list(agt.mod))
agt.pred = predict(agt.mod, x.test)
agt_test_mse = agt.loss(agt.mod, x.test, y.test)
sub_agt_test_loss = c(sub_agt_test_loss, list(agt_test_mse))
agt_train_mse = agt.loss(agt.mod, x.train, y.train)
sub_agt_train_loss = c(sub_agt_train_loss, list(agt_train_mse))
agt_est_gen_loss <- sapply(1:agt.mod$get_num_trees(), agt.mod$estimate_generalization_loss)
sub_agt_estgen_loss <- c(sub_agt_estgen_loss, list(agt_est_gen_loss))
res_mat[b, 1] <- loss_mse(y.test, sam.pred)
res_mat[b, 2] <- loss_mse(y.test, agt.pred)
res[[i]] <- res_mat
setTxtProgressBar(pb, ((i*10)+b-10))
}
sam_models = c(sam_models, list(sub_models))
agt_models = c(agt_models, list(sub_agt_models))
sam_test_loss = c(sam_test_loss, list(sub_test_loss))
agt_test_loss = c(agt_test_loss, list(sub_agt_test_loss))
agt_train_loss = c(agt_train_loss, list(sub_agt_train_loss))
agt_estgen_loss <- c(agt_estgen_loss, list(sub_agt_estgen_loss))
}##
|
| | 0%
|
|= | 1%
|
|== | 3%
|
|=== | 4%
|
|==== | 6%
|
|===== | 7%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|========= | 13%
|
|========== | 14%
|
| | 0%
|
|=========== | 16%
|
|============ | 17%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|================ | 23%
|
|================= | 24%
|
|================== | 26%
|
|=================== | 27%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|======================= | 33%
|
|======================== | 34%
|
|========================= | 36%
|
|========================== | 37%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|================================ | 46%
|
|================================= | 47%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|===================================== | 53%
|
|====================================== | 54%
|
|======================================= | 56%
|
|======================================== | 57%
|
| | 0%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
| | 0%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|===================================================================== | 99%
|
|======================================================================| 100%Comparing train, test and estgen loss for agt and agt sampling to make sure they are similar.
The results are very similar, Hopefully it is the same model with some random noise.
sets = c(1,2,3,4,5,6,7) #dataset, between 1 and 7
iters = c(1, 5, 10) #seed, between 1 and 10
for(i in sets){
for(b in iters){
plot((sam_models[[i]][[b]]$train.loss), main=paste("dataset: ", i), cex = 1.5,
ylab = "Loss", xlab = "iter", type = "l")
lines((sam_test_loss[[i]][[b]]), col = "green", cex = 1.5)
lines((sam_models[[i]][[b]]$gen.loss), col = "red", cex = 1.5)
lines((agt_estgen_loss[[i]][[b]]),col = "brown", cex = 0.8)
lines((agt_test_loss[[i]][[b]]),col = "blue", cex = 0.8)
lines((agt_train_loss[[i]][[b]]),col = "pink", cex = 0.8)
legend( x="topright",
legend=c("Sam Train MSE","Sam Test MSE", "Sam Estgen loss", "Agt Estgen loss", "agt Test MSE", "agt Train MSE"),
col=c("black","green", "red", "brown", "blue", "pink"), lwd=1, lty=c(1))
}
}
training the default model for different sample sizes
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = c(0.4,0.7,0.9), "nrounds"=1000)
sampled_sam_models = list() #List of all sampling models
sampled_sam_test_loss = list() #List of all test loss for sampling models
sampled_sam_res = list()
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7*length(param$samSize), style = 3)
sub_models = list()
sub_test_loss = list()
res_mat <- matrix(nrow=B, ncol=length(param$samSize))
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
sub_sub_models = list()
sub_sub_test_loss = list()
j = 1
for (s in param$samSize){
set.seed(seeds[b])
mod = sampling.agt.train(x.train, y.train, samSize = s,
learnRate = param$learning_rate, Nrounds = param$nrounds,
force_continued_learning = F)
sub_sub_models = c(sub_sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_sub_test_loss = c(sub_sub_test_loss, list(test_mse))
res_mat[b,j] <- loss_mse(y.test, sam.pred)
j = j+1
setTxtProgressBar(pb, (((i-1)*B*length(param$samSize))+(b-1)*length(param$samSize)+j))
}
sub_models = c(sub_models, list(sub_sub_models))
sub_test_loss = c(sub_test_loss, list(sub_sub_test_loss))
sampled_sam_res[[i]] <- res_mat
}
sampled_sam_models = c(sampled_sam_models, list(sub_models))
sampled_sam_test_loss = c(sampled_sam_test_loss, list(sub_test_loss))
}##
|
| | 0%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|======== | 12%
|
|========= | 12%
|
|========= | 13%
|
|========== | 14%
|
|========== | 15%
|
| | 0%
|
|=========== | 15%
|
|=========== | 16%
|
|============ | 17%
|
|============ | 18%
|
|============= | 18%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|=============== | 22%
|
|================ | 22%
|
|================ | 23%
|
|================= | 24%
|
|================= | 25%
|
|================== | 25%
|
|================== | 26%
|
|=================== | 27%
|
|=================== | 28%
|
|==================== | 28%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|====================== | 32%
|
|======================= | 32%
|
|======================= | 33%
|
|======================== | 34%
|
|======================== | 35%
|
|========================= | 35%
|
|========================= | 36%
|
|========================== | 37%
|
|========================== | 38%
|
|=========================== | 38%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================= | 42%
|
|============================== | 42%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|=============================== | 45%
|
|================================ | 45%
|
|================================ | 46%
|
|================================= | 47%
|
|================================= | 48%
|
|================================== | 48%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|==================================== | 52%
|
|===================================== | 52%
|
|===================================== | 53%
|
|====================================== | 54%
|
|====================================== | 55%
|
|======================================= | 55%
|
|======================================= | 56%
|
|======================================== | 57%
|
|======================================== | 58%
|
| | 0%
|
|========================================= | 58%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|=========================================== | 62%
|
|============================================ | 62%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================= | 65%
|
|============================================== | 65%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|=============================================== | 68%
|
|================================================ | 68%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
|================================================== | 72%
|
| | 0%
|
|=================================================== | 72%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|==================================================== | 75%
|
|===================================================== | 75%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|====================================================== | 78%
|
|======================================================= | 78%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================= | 82%
|
|========================================================== | 82%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|=========================================================== | 85%
|
|============================================================ | 85%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================= | 88%
|
|============================================================== | 88%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================ | 92%
|
|================================================================= | 92%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|================================================================== | 95%
|
|=================================================================== | 95%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|==================================================================== | 98%
|
|===================================================================== | 98%
|
|===================================================================== | 99%
|
|======================================================================| 100%results for sampled default
We see that for the default function with sampling size less than 1, we have too much variance in the estimated generalization loss, so the ensembles stop way too early
plot.test.loss <- function(test_loss){
sets = c(1,2,3,4,5,6,7) #dataset, between 1 and 7
iters = c(1,2,3) #seed, between 1 and 10
start = 50
for(i in sets){
for(b in iters){
plot((test_loss[[i]][[b]][[3]]), col = "purple", cex = 0.8, type = "l",
main = paste("Test loss for Dataset: ", i, " Iteration: ", b))
lines(((test_loss[[i]][[b]][[2]])), col = "blue", cex = 0.8)
lines(((test_loss[[i]][[b]][[1]])), col = "green", cex = 0.8)
lines((agt_test_loss[[i]][[b]]), col = "brown", cex = 1.2)
legend( x="topright",
legend=c("AGT", "SAM(0.4)", "SAM(0.7)", "SAM(0.9"),
col=c("brown","green", "blue", "purple"), lwd=1, lty=c(1))
}
}
}
plot.test.loss(sampled_sam_test_loss)
Quick look on estgenloss
plot.gen.loss <- function(models){
sets = c(1,2,3,4,5,6,7) #dataset, between 1 and 7
iters = c(1,2,3) #seed, between 1 and 10
start = 50
for(i in sets){
for(b in iters){
plot((models[[i]][[b]][[3]]$gen.loss), col = "purple", cex = 0.8, type = "l",
main = paste("gen loss for Dataset: ", i, " Iteration: ", b))
lines((models[[i]][[b]][[2]]$gen.loss), col = "blue", cex = 0.8)
lines((models[[i]][[b]][[1]]$gen.loss), col = "green", cex = 0.8)
legend( x="topright",
legend=c("SAM(0.4)", "SAM(0.7)","SAM(0.9)"),
col=c("green", "blue", "purple"), lwd=1, lty=c(1))
}
}
}
plot.gen.loss(sampled_sam_models)
#First alternative Function
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = c(0.4, 0.7, 0.9), "nrounds"=1000)
first_sam_models = list() #List of all sampling models
first_sam_test_loss = list() #List of all test loss for sampling models
first_sam_res = list()
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7*length(param$samSize), style = 3)
sub_models = list()
sub_test_loss = list()
res_mat <- matrix(nrow=B, ncol=length(param$samSize))
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
sub_sub_models = list()
sub_sub_test_loss = list()
j = 1
for (s in param$samSize){
set.seed(seeds[b])
mod = first.sampling.agt.train(x.train, y.train, samSize = s,
learnRate = param$learning_rate, Nrounds = param$nrounds,
force_continued_learning = F)
sub_sub_models = c(sub_sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_sub_test_loss = c(sub_sub_test_loss, list(test_mse))
res_mat[b,j] <- loss_mse(y.test, sam.pred)
j = j+1
setTxtProgressBar(pb, (((i-1)*B*length(param$samSize))+(b-1)*length(param$samSize)+j))
}
sub_models = c(sub_models, list(sub_sub_models))
sub_test_loss = c(sub_test_loss, list(sub_sub_test_loss))
first_sam_res[[i]] <- res_mat
}
first_sam_models = c(first_sam_models, list(sub_models))
first_sam_test_loss = c(first_sam_test_loss, list(sub_test_loss))
}##
|
| | 0%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|======== | 12%
|
|========= | 12%
|
|========= | 13%
|
|========== | 14%
|
|========== | 15%
|
| | 0%
|
|=========== | 15%
|
|=========== | 16%
|
|============ | 17%
|
|============ | 18%
|
|============= | 18%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|=============== | 22%
|
|================ | 22%
|
|================ | 23%
|
|================= | 24%
|
|================= | 25%
|
|================== | 25%
|
|================== | 26%
|
|=================== | 27%
|
|=================== | 28%
|
|==================== | 28%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|====================== | 32%
|
|======================= | 32%
|
|======================= | 33%
|
|======================== | 34%
|
|======================== | 35%
|
|========================= | 35%
|
|========================= | 36%
|
|========================== | 37%
|
|========================== | 38%
|
|=========================== | 38%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================= | 42%
|
|============================== | 42%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|=============================== | 45%
|
|================================ | 45%
|
|================================ | 46%
|
|================================= | 47%
|
|================================= | 48%
|
|================================== | 48%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|==================================== | 52%
|
|===================================== | 52%
|
|===================================== | 53%
|
|====================================== | 54%
|
|====================================== | 55%
|
|======================================= | 55%
|
|======================================= | 56%
|
|======================================== | 57%
|
|======================================== | 58%
|
| | 0%
|
|========================================= | 58%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|=========================================== | 62%
|
|============================================ | 62%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================= | 65%
|
|============================================== | 65%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|=============================================== | 68%
|
|================================================ | 68%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
|================================================== | 72%
|
| | 0%
|
|=================================================== | 72%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|==================================================== | 75%
|
|===================================================== | 75%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|====================================================== | 78%
|
|======================================================= | 78%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================= | 82%
|
|========================================================== | 82%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|=========================================================== | 85%
|
|============================================================ | 85%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================= | 88%
|
|============================================================== | 88%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================ | 92%
|
|================================================================= | 92%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|================================================================== | 95%
|
|=================================================================== | 95%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|==================================================================== | 98%
|
|===================================================================== | 98%
|
|===================================================================== | 99%
|
|======================================================================| 100%results for sampled first
Not very promising
plot.test.loss(first_sam_test_loss)
plot.results <- function(results){
sets = c(1,2,3,4,5,6,7) #dataset, between 1 and 7
plot(1,1, xlim = c(0.35, 0.95), ylim = c(0.75,1.5),
main = "mean Test loss relative to AGT for different samSize on each dataset",
ylab = "loss", xlab = "sampling size")
for(i in sets){
lines(x = c(0.4,0.7,0.9), y = colMeans(results[[i]])/mean(res[[i]][,2]), col = i, xlim = c(0.3, 1))
abline(h=1)
legend( x="topright",
legend=c("1", "2","3", "4", "5", "6", "7"),
col=c(1,2,3,4,5,6,7), lwd=1, lty=c(1))
}
}
plot.results(first_sam_res)
Quick look on estgenloss
plot.gen.loss(first_sam_models)
Min function
Stops if the min estgenloss is not in the last a iterations
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = c(0.4,0.7,0.9), "nrounds"=1000)
min_sam_models = list() #List of all sampling models
min_sam_test_loss = list() #List of all test loss for sampling models
min_sam_res = list()
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7*length(param$samSize), style = 3)
sub_models = list()
sub_test_loss = list()
res_mat <- matrix(nrow=B, ncol=length(param$samSize))
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
sub_sub_models = list()
sub_sub_test_loss = list()
j = 1
for (s in param$samSize){
set.seed(seeds[b])
mod = min.sampling.agt.train(x.train, y.train, samSize = s, a=50,
learnRate = param$learning_rate, Nrounds = param$nrounds,
force_continued_learning = F)
sub_sub_models = c(sub_sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_sub_test_loss = c(sub_sub_test_loss, list(test_mse))
res_mat[b,j] <- loss_mse(y.test, sam.pred)
j = j+1
setTxtProgressBar(pb, (((i-1)*B*length(param$samSize))+(b-1)*length(param$samSize)+j))
}
sub_models = c(sub_models, list(sub_sub_models))
sub_test_loss = c(sub_test_loss, list(sub_sub_test_loss))
min_sam_res[[i]] <- res_mat
}
min_sam_models = c(min_sam_models, list(sub_models))
min_sam_test_loss = c(min_sam_test_loss, list(sub_test_loss))
}##
|
| | 0%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|======== | 12%
|
|========= | 12%
|
|========= | 13%
|
|========== | 14%
|
|========== | 15%
|
| | 0%
|
|=========== | 15%
|
|=========== | 16%
|
|============ | 17%
|
|============ | 18%
|
|============= | 18%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|=============== | 22%
|
|================ | 22%
|
|================ | 23%
|
|================= | 24%
|
|================= | 25%
|
|================== | 25%
|
|================== | 26%
|
|=================== | 27%
|
|=================== | 28%
|
|==================== | 28%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|====================== | 32%
|
|======================= | 32%
|
|======================= | 33%
|
|======================== | 34%
|
|======================== | 35%
|
|========================= | 35%
|
|========================= | 36%
|
|========================== | 37%
|
|========================== | 38%
|
|=========================== | 38%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================= | 42%
|
|============================== | 42%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|=============================== | 45%
|
|================================ | 45%
|
|================================ | 46%
|
|================================= | 47%
|
|================================= | 48%
|
|================================== | 48%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|==================================== | 52%
|
|===================================== | 52%
|
|===================================== | 53%
|
|====================================== | 54%
|
|====================================== | 55%
|
|======================================= | 55%
|
|======================================= | 56%
|
|======================================== | 57%
|
|======================================== | 58%
|
| | 0%
|
|========================================= | 58%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|=========================================== | 62%
|
|============================================ | 62%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================= | 65%
|
|============================================== | 65%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|=============================================== | 68%
|
|================================================ | 68%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
|================================================== | 72%
|
| | 0%
|
|=================================================== | 72%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|==================================================== | 75%
|
|===================================================== | 75%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|====================================================== | 78%
|
|======================================================= | 78%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================= | 82%
|
|========================================================== | 82%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|=========================================================== | 85%
|
|============================================================ | 85%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================= | 88%
|
|============================================================== | 88%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================ | 92%
|
|================================================================= | 92%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|================================================================== | 95%
|
|=================================================================== | 95%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|==================================================================== | 98%
|
|===================================================================== | 98%
|
|===================================================================== | 99%
|
|======================================================================| 100%results for sampled min
The results are good, but I am including 50 trees after the minimum estgenloss, so the method is not very good. potential bug
plot.test.loss(min_sam_test_loss)
plot.results(min_sam_res)
Quick look on estgenloss
plot.gen.loss(min_sam_models)
### plotting training loss
sets = c(1,2,3,4,5,6,7) #dataset, between 1 and 7
iters = c(1,2,3) #seed, between 1 and 10
start = 50
for(i in sets){
for(b in iters){
plot((min_sam_models[[i]][[b]][[3]]$train.loss), col = "purple", cex = 0.8, type = "l",
main = paste("Train loss for Dataset: ", i, " Iteration: ", b))
lines((min_sam_models[[i]][[b]][[2]]$train.loss), col = "blue", cex = 0.8)
lines((min_sam_models[[i]][[b]][[1]]$train.loss), col = "green", cex = 0.8)
legend( x="topright",
legend=c("SAM(0.4)", "SAM(0.7)","SAM(0.9)"),
col=c("green", "blue", "purple"), lwd=1, lty=c(1))
}
}
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = c(0.4,0.7,0.9), "nrounds"=1000)
full_sam_models = list() #List of all sampling models
full_sam_test_loss = list() #List of all test loss for sampling models
full_sam_res = list()
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7*length(param$samSize), style = 3)
sub_models = list()
sub_test_loss = list()
res_mat <- matrix(nrow=B, ncol=length(param$samSize))
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
sub_sub_models = list()
sub_sub_test_loss = list()
j = 1
for (s in param$samSize){
set.seed(seeds[b])
mod = full.sampling.agt.train(x.train, y.train, samSize = s,
learnRate = param$learning_rate, Nrounds = param$nrounds,
force_continued_learning = F)
sub_sub_models = c(sub_sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_sub_test_loss = c(sub_sub_test_loss, list(test_mse))
res_mat[b,j] <- loss_mse(y.test, sam.pred)
j = j+1
setTxtProgressBar(pb, (((i-1)*B*length(param$samSize))+(b-1)*length(param$samSize)+j))
}
sub_models = c(sub_models, list(sub_sub_models))
sub_test_loss = c(sub_test_loss, list(sub_sub_test_loss))
full_sam_res[[i]] <- res_mat
}
full_sam_models = c(full_sam_models, list(sub_models))
full_sam_test_loss = c(full_sam_test_loss, list(sub_test_loss))
}##
|
| | 0%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|======== | 12%
|
|========= | 12%
|
|========= | 13%
|
|========== | 14%
|
|========== | 15%
|
| | 0%
|
|=========== | 15%
|
|=========== | 16%
|
|============ | 17%
|
|============ | 18%
|
|============= | 18%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|=============== | 22%
|
|================ | 22%
|
|================ | 23%
|
|================= | 24%
|
|================= | 25%
|
|================== | 25%
|
|================== | 26%
|
|=================== | 27%
|
|=================== | 28%
|
|==================== | 28%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|====================== | 32%
|
|======================= | 32%
|
|======================= | 33%
|
|======================== | 34%
|
|======================== | 35%
|
|========================= | 35%
|
|========================= | 36%
|
|========================== | 37%
|
|========================== | 38%
|
|=========================== | 38%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================= | 42%
|
|============================== | 42%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|=============================== | 45%
|
|================================ | 45%
|
|================================ | 46%
|
|================================= | 47%
|
|================================= | 48%
|
|================================== | 48%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|==================================== | 52%
|
|===================================== | 52%
|
|===================================== | 53%
|
|====================================== | 54%
|
|====================================== | 55%
|
|======================================= | 55%
|
|======================================= | 56%
|
|======================================== | 57%
|
|======================================== | 58%
|
| | 0%
|
|========================================= | 58%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|=========================================== | 62%
|
|============================================ | 62%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================= | 65%
|
|============================================== | 65%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|=============================================== | 68%
|
|================================================ | 68%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
|================================================== | 72%
|
| | 0%
|
|=================================================== | 72%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|==================================================== | 75%
|
|===================================================== | 75%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|====================================================== | 78%
|
|======================================================= | 78%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================= | 82%
|
|========================================================== | 82%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|=========================================================== | 85%
|
|============================================================ | 85%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================= | 88%
|
|============================================================== | 88%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================ | 92%
|
|================================================================= | 92%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|================================================================== | 95%
|
|=================================================================== | 95%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|==================================================================== | 98%
|
|===================================================================== | 98%
|
|===================================================================== | 99%
|
|======================================================================| 100%results for sampled full
plot.test.loss(full_sam_test_loss)
plot.results(full_sam_res)
Quick look on estgenloss
gen loss is off here because of a fixed bug
plot.gen.loss(full_sam_models)
set.seed(1295)
seed = 1295
B = 10
seeds <- sample(1e5, B)
param <- list("learning_rate" = 0.1, "samSize" = c(0.4,0.7,0.9), "nrounds"=1000)
same_sam_models = list() #List of all sampling models
same_sam_test_loss = list() #List of all test loss for sampling models
same_sam_res = list()
for(i in 1:7){
pb <- txtProgressBar(min = 0, max = B*7*length(param$samSize), style = 3)
sub_models = list()
sub_test_loss = list()
res_mat <- matrix(nrow=B, ncol=length(param$samSize))
for (b in 1:B){
#cat("iter: ", i,"\n")
dataset(i, seeds[b])
sub_sub_models = list()
sub_sub_test_loss = list()
j = 1
for (s in param$samSize){
set.seed(seeds[b])
mod = same.sampling.agt.train(x.train, y.train, samSize = s,
learnRate = param$learning_rate, Nrounds = param$nrounds,
force_continued_learning = F)
sub_sub_models = c(sub_sub_models, list(mod))
sam.pred = sampling.agt.pred(mod, x.test)
test_mse = sampling.agt.loss(mod, x.test, y.test)
sub_sub_test_loss = c(sub_sub_test_loss, list(test_mse))
res_mat[b,j] <- loss_mse(y.test, sam.pred)
j = j+1
setTxtProgressBar(pb, (((i-1)*B*length(param$samSize))+(b-1)*length(param$samSize)+j))
}
sub_models = c(sub_models, list(sub_sub_models))
sub_test_loss = c(sub_test_loss, list(sub_sub_test_loss))
same_sam_res[[i]] <- res_mat
}
same_sam_models = c(same_sam_models, list(sub_models))
same_sam_test_loss = c(same_sam_test_loss, list(sub_test_loss))
}##
|
| | 0%
|
|= | 1%
|
|= | 2%
|
|== | 2%
|
|== | 3%
|
|=== | 4%
|
|=== | 5%
|
|==== | 5%
|
|==== | 6%
|
|===== | 7%
|
|===== | 8%
|
|====== | 8%
|
|====== | 9%
|
|======= | 10%
|
|======== | 11%
|
|======== | 12%
|
|========= | 12%
|
|========= | 13%
|
|========== | 14%
|
|========== | 15%
|
| | 0%
|
|=========== | 15%
|
|=========== | 16%
|
|============ | 17%
|
|============ | 18%
|
|============= | 18%
|
|============= | 19%
|
|============== | 20%
|
|=============== | 21%
|
|=============== | 22%
|
|================ | 22%
|
|================ | 23%
|
|================= | 24%
|
|================= | 25%
|
|================== | 25%
|
|================== | 26%
|
|=================== | 27%
|
|=================== | 28%
|
|==================== | 28%
|
|==================== | 29%
|
| | 0%
|
|===================== | 30%
|
|====================== | 31%
|
|====================== | 32%
|
|======================= | 32%
|
|======================= | 33%
|
|======================== | 34%
|
|======================== | 35%
|
|========================= | 35%
|
|========================= | 36%
|
|========================== | 37%
|
|========================== | 38%
|
|=========================== | 38%
|
|=========================== | 39%
|
|============================ | 40%
|
|============================= | 41%
|
|============================= | 42%
|
|============================== | 42%
|
|============================== | 43%
|
| | 0%
|
|=============================== | 44%
|
|=============================== | 45%
|
|================================ | 45%
|
|================================ | 46%
|
|================================= | 47%
|
|================================= | 48%
|
|================================== | 48%
|
|================================== | 49%
|
|=================================== | 50%
|
|==================================== | 51%
|
|==================================== | 52%
|
|===================================== | 52%
|
|===================================== | 53%
|
|====================================== | 54%
|
|====================================== | 55%
|
|======================================= | 55%
|
|======================================= | 56%
|
|======================================== | 57%
|
|======================================== | 58%
|
| | 0%
|
|========================================= | 58%
|
|========================================= | 59%
|
|========================================== | 60%
|
|=========================================== | 61%
|
|=========================================== | 62%
|
|============================================ | 62%
|
|============================================ | 63%
|
|============================================= | 64%
|
|============================================= | 65%
|
|============================================== | 65%
|
|============================================== | 66%
|
|=============================================== | 67%
|
|=============================================== | 68%
|
|================================================ | 68%
|
|================================================ | 69%
|
|================================================= | 70%
|
|================================================== | 71%
|
|================================================== | 72%
|
| | 0%
|
|=================================================== | 72%
|
|=================================================== | 73%
|
|==================================================== | 74%
|
|==================================================== | 75%
|
|===================================================== | 75%
|
|===================================================== | 76%
|
|====================================================== | 77%
|
|====================================================== | 78%
|
|======================================================= | 78%
|
|======================================================= | 79%
|
|======================================================== | 80%
|
|========================================================= | 81%
|
|========================================================= | 82%
|
|========================================================== | 82%
|
|========================================================== | 83%
|
|=========================================================== | 84%
|
|=========================================================== | 85%
|
|============================================================ | 85%
|
|============================================================ | 86%
|
| | 0%
|
|============================================================= | 87%
|
|============================================================= | 88%
|
|============================================================== | 88%
|
|============================================================== | 89%
|
|=============================================================== | 90%
|
|================================================================ | 91%
|
|================================================================ | 92%
|
|================================================================= | 92%
|
|================================================================= | 93%
|
|================================================================== | 94%
|
|================================================================== | 95%
|
|=================================================================== | 95%
|
|=================================================================== | 96%
|
|==================================================================== | 97%
|
|==================================================================== | 98%
|
|===================================================================== | 98%
|
|===================================================================== | 99%
|
|======================================================================| 100%results for sampled same
plot.test.loss(same_sam_test_loss)
plot.results(same_sam_res)
Quick look on estgenloss
plot.gen.loss(same_sam_models)
